// things to note for MPI+pthread:
// when main ends, it kills the threads
// when any thread mpi_finalize()s in 1 cpu, MPI thinks this cpu finalizes
// need to be careful with shared memory


#include <stdio.h>
#include <stdlib.h>
#include <pthread.h>
#include "mpi.h"

#define TAG 1

MPI_Status status;
MPI_Comm UO_COMM;
int wsize;
int wself;
int usize;
int uself;

void *startThread(void *threadid)
{
	int tid;
	tid = (int)threadid;
	int values[2];
	int sums[2];
	int flag[4];
	flag[0] = 0;
	flag[1] = 0;
	flag[2] = 0;
	flag[3] = 0;
	
	//printf("Thread %d started\n", tid);
	//if (uself == 0 && tid == 0) {
	//	printf("Prepare to send...\n");
	//	int sum = 10;
	//	int i = 1;
		//for(i=0; i<2; i++) {
	//		MPI_Send(&sum, 1, MPI_INT, i, TAG, UO_COMM);
		//}
	//}
	// 0 for world
	// 1 for uo
	
	if (tid == 0) {
		printf("Rank %d Thread %d: waiting...\n", wself, tid);
		MPI_Recv(&values[tid], 1, MPI_INT, MPI_ANY_SOURCE, 1, MPI_COMM_WORLD, &status);
		printf("Rank %d Thread %d: value: %d\n", wself, tid, values[tid]);
		flag[wself*2+tid] = 1;
	} else {
		printf("Rank %d Thread %d: waiting...\n", uself, tid);
		MPI_Recv(&values[tid], 1, MPI_INT, MPI_ANY_SOURCE, TAG, UO_COMM, &status);
		printf("Rank %d Thread %d: value: %d\n", uself, tid, values[tid]);
		flag[uself*2+tid] = 1;
	}
	
	printf("Thread %d: Finalizing...\n", tid);
	printf("%d, %d, %d, %d\n", flag[0], flag[1], flag[2], flag[3]);
	if (flag[0] == 1 && flag[1] == 1 && flag[2] == 1 && flag[3] == 1) {
		MPI_Finalize();
	}
	
	return;
}

int main(int argc,char **argv)
{
	int split_color = 1;
	int split_rank = 1;
	
	MPI_Init(&argc, &argv);
	
	MPI_Comm_split(MPI_COMM_WORLD, split_color, split_rank, &UO_COMM);
	
	MPI_Comm_size(MPI_COMM_WORLD, &wsize);
	MPI_Comm_rank(MPI_COMM_WORLD, &wself);
	
	MPI_Comm_size(UO_COMM, &usize);
	MPI_Comm_rank(UO_COMM, &uself);
	
	//printf("self: %d, size: %d\n", wself, wsize);
	//printf("self: %d, size: %d\n", uself, usize);
	
	pthread_t threads[2];
	int i;
	for(i=0; i<2; i++) {
		pthread_create(&threads[i], NULL, startThread, (void *)i);
	}
	
	sleep(3);
	
	if (uself == 0) {
		printf("Prepare to send...\n");
		int sum = 10;
		for(i=0; i<2; i++) {
			MPI_Send(&sum, 1, MPI_INT, i, TAG, MPI_COMM_WORLD);
		}
	}
	//sleep(3);
	//sum = 4;
	//for(i=0; i<2; i++) {
	//	MPI_Send(&sum, 1, MPI_INT, i, TAG, MPI_COMM_WORLD);
	//}
	
	sleep(10);
	//MPI_Barrier(UO_COMM);
	//MPI_Barrier(MPI_COMM_WORLD);
	
	//MPI_Finalize();
	
	return 0;
}



